﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data;
using App.Core.Extensions;
using HtmlAgilityPack;
using App.Core.DB;
using WebScrapper.Helpers;

namespace WebScrapper.Implementations.Alice
{
    public class Manta : BaseImplementation<App.Core.Base.Model.BaseEntity>
    {
        private const String DOMAIN = "http://www.manta.com";
        private DataTable _Dt = new DataTable("Companies");
        
        #region Data Table Columns
        private const String NAME = "Company Name";
        //private const String DIGREE = "Digree";
        private const String DESCRIPTION = "Description";
        //private const String CLINIC_NAME = "Clinic Name";
        private const String PHONE = "Phone";
        private const String FAX = "Fax";
        private const String EMAIL = "Email";
        private const String WEB_SITE = "Web Site";
        private const String STREET_ADDRESS = "Street Address";        
        private const String LOCALITY = "Locality";
        private const String REGION = "Region";        
        private const String POSTAL_CODE = "Postal Code";
        private const String ANNUAL_SALES = "Annual Sales";
        private const String ANNUAL_REVENUE = "Annual Revenue";
        private const String ANNUAL_SALES_OR_REVENUE = "Annual Sales Or Revenue";        
        private const String EMPLOYEES = "Employees";
        private const String YEARS_IN_BUSINESS = "Years In Business";
        private const String CONTACT1 = "Contact1";
        private const String CONTACT2 = "Contact2";
        private const String CONTACT3 = "Contact3";
        private const String CONTACT4 = "Contact4";
        private const String CONTACT5 = "Contact5";
        private const String CONTACT6 = "Contact6";
        private static String DETAILS_URL = "Details Url";
        

        #endregion

        public Manta(System.Windows.Forms.RichTextBox txtStatus)
            : base(txtStatus)
        {
            ///Initialize the Scrapped DataTable With Columns
            _Dt.Columns.Add(new DataColumn(NAME));
            //_Dt.Columns.Add(new DataColumn(DIGREE));
            _Dt.Columns.Add(new DataColumn(DESCRIPTION));
            //_Dt.Columns.Add(new DataColumn(CLINIC_NAME));
            _Dt.Columns.Add(new DataColumn(PHONE));
            _Dt.Columns.Add(new DataColumn(FAX));
            _Dt.Columns.Add(new DataColumn(EMAIL));
            _Dt.Columns.Add(new DataColumn(STREET_ADDRESS));
            _Dt.Columns.Add(new DataColumn(LOCALITY));
            _Dt.Columns.Add(new DataColumn(REGION));
            _Dt.Columns.Add(new DataColumn(POSTAL_CODE));
            _Dt.Columns.Add(new DataColumn(WEB_SITE));
            _Dt.Columns.Add(new DataColumn(ANNUAL_SALES));
            _Dt.Columns.Add(new DataColumn(ANNUAL_REVENUE));
            _Dt.Columns.Add(new DataColumn(ANNUAL_SALES_OR_REVENUE));
            _Dt.Columns.Add(new DataColumn(CONTACT1));
            _Dt.Columns.Add(new DataColumn(CONTACT2));
            _Dt.Columns.Add(new DataColumn(CONTACT3));
            _Dt.Columns.Add(new DataColumn(CONTACT4));
            _Dt.Columns.Add(new DataColumn(CONTACT5));
            _Dt.Columns.Add(new DataColumn(CONTACT6));
            _Dt.Columns.Add(new DataColumn(EMPLOYEES));
            _Dt.Columns.Add(new DataColumn(YEARS_IN_BUSINESS));
            _Dt.Columns.Add(new DataColumn(DETAILS_URL));
        }
        public override void StartScrapping(string category)
        {
            String urlFormat = "http://www.manta.com/mb_45_B82C70B4_01/heating_and_air_conditioning_contractors/alabama?pg={0}";
            for (int i = 1; i <= 16; i++)
            {
                String url = String.Format(urlFormat, i);
                UpdateStatusText(String.Format("Starting to process State '{0}'", url));
                ProcessState(url);
                UpdateStatusText(String.Format("Processed State Successfully. {0}", Environment.NewLine));

                String fileName = String.Format("Alabama_{0}.xls", i);
                String filePath = DataTableToExcel(_Dt, null, true, fileName);
                UpdateStatusText(String.Format("Output File has been generated in '{0}'", filePath));
            }            
        }

        private void ProcessState(string url)
        {
            String html = GetAuthenticatedPageContent(url, ConfigReader.AuthenticationCookieValue);
            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
            doc.LoadHtml(html);
            ///TODO: For testing
            //doc.Load(@"C:\Users\Mimi\Downloads\Manta\alabama.htm");
            HtmlNode listingUl = doc.DocumentNode.SelectSingleNode("//ul[@class='results-list tab-switch-content tab-switch-content-shown']");
            HtmlNodeCollection lis = listingUl.SelectNodes("li");
            //HtmlNodeCollection lis = listingUl.ChildNodes;
            
            for (int i = 0; i < lis.Count; i ++)
            {
                HtmlNode li = lis[i];
                if (IsCompanyInfoLi(li))
                {
                    HtmlNode divNode = li.SelectNodes("div[@class='result-box vcard']")[0];
                    HtmlNode divFnOrg = divNode.SelectSingleNode("div[@class='fn org']");
                    HtmlNode companyAnchor = divFnOrg.SelectNodes("h2")[0].SelectNodes("a")[0];
                    DataRow row = _Dt.NewRow();
                    row[NAME] = CleanUpText(companyAnchor.InnerText.HtmlDecode());
                    row[DETAILS_URL] = companyAnchor.Attributes["href"].Value;                    
                    LoadDetails(row);
                    _Dt.Rows.Add(row);
                    //if(i > 1)
                    //    break;
                    System.Threading.Thread.Sleep(5000);
                }
            }            
        }

        private void LoadDetails(DataRow row)
        {
            String detailsUrl = NullHandler.GetString(row[DETAILS_URL]);
            if (!detailsUrl.IsNullOrEmpty())
            {
                UpdateStatusText(String.Format("Loading Details from '{0}'", detailsUrl));
                String html = String.Empty;
                HtmlDocument doc = new HtmlDocument();
                HtmlWeb hw = new HtmlWeb();
                try
                {
                    html = GetAuthenticatedPageContent(detailsUrl, ConfigReader.AuthenticationCookieValue);
                    //doc = hw.Load(detailsUrl);
                    doc.LoadHtml(html);
                }
                catch (Exception ex)
                {
                    UpdateStatusText(String.Format("Following Error occured during loading Deatils URL:{0} {1}", Environment.NewLine, ex.Message));
                    return;
                }               
                
                HtmlNode detailsNode = doc.DocumentNode.SelectSingleNode("//div[@id='profile-tab-contents']");
                if (detailsNode == null)
                    UpdateStatusText("Details Not Found.");
                else
                {   
                    HtmlNode divFeaturedProfile = detailsNode.SelectSingleNode("div[@id='profile-holder']");
                    HtmlNode companyInfo = null;
                    if (divFeaturedProfile == null)
                    {
                       companyInfo = detailsNode.SelectSingleNode("div[@class='company-info vcard']");
                       ///Load Telephone
                       HtmlNodeCollection otherInfos = detailsNode.SelectSingleNode("div[@class='company-info vcard']").SelectNodes("dl");
                       if (otherInfos != null && otherInfos.Count > 0)
                       {
                           HtmlNode ddlPhone = otherInfos[0].SelectSingleNode("dd[@class='tel']");
                           row[PHONE] = ddlPhone == null ? String.Empty : CleanUpText(ddlPhone.InnerText);
                           HtmlNode webSite = otherInfos.Count > 1 ? otherInfos[1] : null;
                           if (webSite != null && CleanUpText(webSite.InnerText).IndexOf("Information not found") == -1)
                           {
                               HtmlNode dd = webSite.SelectSingleNode("dd");
                               HtmlNode anchor = dd == null ? null : dd.SelectSingleNode("a");
                               row[WEB_SITE] = anchor == null ? String.Empty : anchor.GetAttributeValue("href", String.Empty);
                           }
                       }
                    }
                    else
                    {
                        companyInfo = divFeaturedProfile.SelectSingleNode("div[@class='company-box']").SelectSingleNode("div[@class='company-info vcard']");
                        HtmlNode phone = companyInfo.SelectSingleNode("p[@class='telephone tel']");
                        row[PHONE] = phone == null ? String.Empty : CleanUpText(phone.InnerText);
                        HtmlNode email = companyInfo.SelectSingleNode("p[@class='email']");
                        row[EMAIL] = email == null ? String.Empty : FormatAsEmail(CleanUpText(email.InnerText));
                        var webSite = companyInfo.Descendants("p")
                        .Where(P => P.GetAttributeValue("class", String.Empty) == String.Empty)
                        .SelectMany(P => P.Descendants("a"));
                        row[WEB_SITE] = webSite == null || webSite.Count() == 0 ? String.Empty : webSite.ToList()[0].Attributes["href"].Value;
                    }
                    HtmlNode divAddress = companyInfo.SelectSingleNode("div[@class='adr']"); 
                    HtmlNode streetAddress = divAddress.SelectSingleNode("div[@class='street-address']");
                    HtmlNode locality = divAddress.SelectSingleNode("span[@class='locality']");
                    HtmlNode region = divAddress.SelectSingleNode("span[@class='region']");
                    HtmlNode postalCode = divAddress.SelectSingleNode("span[@class='postal-code']");

                    row[STREET_ADDRESS] = streetAddress == null ? String.Empty : CleanUpText(streetAddress.InnerText);
                    row[LOCALITY] = locality == null ? String.Empty : CleanUpText(locality.InnerText);
                    row[REGION] = region == null ? String.Empty : CleanUpText(region.InnerText);
                    row[POSTAL_CODE] = postalCode == null ? String.Empty : CleanUpText(postalCode.InnerText);
                    
                    ///Load Description
                    HtmlNode details = detailsNode.SelectSingleNode("div[@class='about']").SelectSingleNode("p");
                    row[DESCRIPTION] = details == null ? String.Empty : CleanUpText(details.InnerText);

                    ///Load Contacts
                    
                    ///Loading Annual Sales
                    UpdateStatusText("Loading Revenue Info");
                    HtmlNode tableData = doc.DocumentNode.SelectSingleNode("//table[@class='table-data']");
                    if (tableData != null)
                    {
                        //HtmlNode tbody = tableData.SelectSingleNode("tbody");
                        HtmlNodeCollection trs = tableData.SelectNodes("tr"); //tbody == null ? null : tbody.SelectNodes("tr");
                        if (trs != null && trs.Count > 0)
                        {
                            if (divFeaturedProfile == null)
                            {
                                LoadNonFeaturedContacts(doc, row);
                                row[ANNUAL_REVENUE] = GetRevenueInfo(trs, "Annual");
                                row[ANNUAL_SALES_OR_REVENUE] = row[ANNUAL_REVENUE];
                            }
                            else
                            {
                                LoadFeaturedContacts(doc, row);
                                row[ANNUAL_SALES] = GetRevenueInfo(trs, "Annual");
                                row[ANNUAL_SALES_OR_REVENUE] = row[ANNUAL_SALES];
                            }

                            row[EMPLOYEES] = GetRevenueInfo(trs, "Employees");
                            row[YEARS_IN_BUSINESS] = GetRevenueInfo(trs, "Years in Business");
                            UpdateStatusText("Revenue Info Loaded Successfully");
                        }
                    }
                }
            }
        }        
        private void LoadNonFeaturedContacts(HtmlDocument doc, DataRow row)
        {
            UpdateStatusText("Loading Contacts");
            HtmlNode ul = doc.DocumentNode.SelectSingleNode("//ul[@class='items']");
            if (ul == null)
                UpdateStatusText("Contacts Not Found");
            else
            {
                HtmlNodeCollection lis = ul.SelectNodes("li");
                if (lis != null && lis.Count > 0)
                {
                    UpdateStatusText(String.Format("Total {0} Contacts Found.", lis.Count));
                    for (int i = 0; i < lis.Count; i++)
                    {
                        HtmlNode li = lis[i];
                        HtmlNode div = li.SelectSingleNode("div");
                        if (div != null)
                        {
                            StringBuilder sb = new StringBuilder(10);
                            HtmlNode name = div.SelectSingleNode("h6");
                            HtmlNode title = div.SelectSingleNode("em");
                            sb.AppendFormat("Name: {0} ({1}) ", name == null ? String.Empty : CleanUpText(name.InnerText), title == null ? String.Empty : CleanUpText(title.InnerText));
                            ///Phones
                            //HtmlNodeCollection phones = divVCard.SelectNodes("p[@class='tel']");

                            //if (phones != null && phones.Count > 0)
                            //{
                            //    sb.Append("Phone: ");
                            //    for (int p = 0; i < phones.Count; p++)
                            //    {
                            //        if (p == 0)
                            //            sb.AppendFormat("{0} ", CleanUpText(phones[p].InnerText));
                            //        else
                            //            sb.AppendFormat(", {0} ", CleanUpText(phones[p].InnerText));
                            //    }
                            //}

                            //HtmlNode email = divVCard.SelectSingleNode("p[@class='email']");
                            //if (email != null)
                            //    sb.AppendFormat("Email: {0}", CleanUpText(email.InnerText));
                            row[String.Format("Contact{0}", i + 1)] = sb.ToString();
                        }
                        ///There is Space for Total 6 Contacts in the Output Excel Sheet
                        if (i == 5)
                            break;
                    }
                }
            }
        }
        private void LoadFeaturedContacts(HtmlDocument doc, DataRow row)
        {            
            UpdateStatusText("Loading Contacts");
            HtmlNode ul = doc.DocumentNode.SelectSingleNode("//ul[@class='items']");
            if (ul == null)
                UpdateStatusText("Contacts Not Found");
            else
            {
                HtmlNodeCollection lis = ul.SelectNodes("li");
                if (lis != null && lis.Count > 0)
                {
                    UpdateStatusText(String.Format("Total {0} Contacts Found.", lis.Count));
                    for (int i = 0; i < lis.Count; i++)
                    {
                        HtmlNode li = lis[i];
                        HtmlNode divVCard = li.SelectSingleNode("div[@class='vcard']");
                        if (divVCard != null)
                        {
                            StringBuilder sb = new StringBuilder(10);
                            HtmlNode name = divVCard.SelectSingleNode("a");
                            HtmlNode title = divVCard.SelectSingleNode("em[@class='title']");
                            sb.AppendFormat("Name: {0} ({1}) ", CleanUpText(name.InnerText), title == null ? String.Empty : CleanUpText(title.InnerText));
                            ///Phones
                            HtmlNodeCollection phones = divVCard.SelectNodes("p[@class='tel']");
                            
                            if (phones != null && phones.Count > 0)
                            {
                                sb.Append("Phone: ");
                                for (int p = 0; p < phones.Count; p++)
                                {
                                    if (p == 0)
                                        sb.AppendFormat("{0} ", CleanUpText(phones[p].InnerText));
                                    else
                                        sb.AppendFormat(", {0} ", CleanUpText(phones[p].InnerText));
                                }                                
                            }

                            HtmlNode email = divVCard.SelectSingleNode("p[@class='email']");
                            if (email != null)
                            {
                                HtmlNode script = email.SelectSingleNode("script");
                                if (script != null)
                                {                                    
                                    //script.Remove();
                                    sb.AppendFormat("Email: {0}", FormatAsEmail(CleanUpText(script.InnerText)));
                                }
                            }
                            row[String.Format("Contact{0}", i + 1)] = sb.ToString();
                        }
                    }
                }
            }
        }

        private String GetRevenueInfo(HtmlNodeCollection trs, String caption)
        {
            foreach (HtmlNode tr in trs)
            {
                HtmlNode td = tr.SelectSingleNode("td");
                HtmlNode th = tr.SelectSingleNode("th");
                if (th != null && CleanUpText(th.InnerText).IndexOf(caption) > -1)
                    return CleanUpText(td.InnerText);
            }
            return String.Empty;
        }

        private bool IsCompanyInfoLi(HtmlNode li)
        {
            String[] nonCompanyLiClasses = {"adsense-unit", "promo", "image-area"};
            if (String.Compare(li.Name, "li", true) != 0)
                return false;
            else if (li.Attributes.Count > 0)
            {
                if (nonCompanyLiClasses.Contains(li.Attributes["class"].Value))
                    return false;
            }
            return true;
        }
        private string FormatAsEmail(String emailText)
        {
            if (!emailText.IsNullOrEmpty())
            {
                String[] parts = emailText.Split(';');
                if (parts.Length > 2)
                {
                    String first = parts[0].Replace("wr(\"", String.Empty).Replace("\")", String.Empty);
                    String last = parts[2].Replace("wr(\"", String.Empty).Replace("\")", String.Empty);
                    return String.Format("{0}@{1}", first, last);
                }
            }
            return String.Empty;
        }
        public override string OutputFile(List<App.Core.Base.Model.BaseEntity> collection, bool addColumnHeader, string outputFileName)
        {
            throw new NotImplementedException();
        }
    }
}
